From: Jan Beulich Date: Fri, 9 Mar 2018 16:29:45 +0000 (+0100) Subject: x86: improve MSR_SHADOW_GS accesses X-Git-Tag: archive/raspbian/4.11.1-1+rpi1~1^2~66^2~434 X-Git-Url: https://dgit.raspbian.org/%22http://www.example.com/cgi/%22/%22http:/www.example.com/cgi/%22?a=commitdiff_plain;h=b78a43d60af39a8a3d59ddc2e33d5aa290f5a7f9;p=xen.git x86: improve MSR_SHADOW_GS accesses Instead of using RDMSR/WRMSR, on fsgsbase-capable systems use a double SWAPGS combined with RDGSBASE/WRGSBASE. This halves execution time for a shadow GS update alone on my Haswell (and we have indications of good performance improvements by this on Skylake too), while the win is even higher when e.g. updating more than one base (as may and commonly will happen in load_segments()). Signed-off-by: Jan Beulich Reviewed-by: Andrew Cooper Reviewed-by: Kevin Tian --- diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c index 69679a6525..b4e062472e 100644 --- a/xen/arch/x86/domain.c +++ b/xen/arch/x86/domain.c @@ -1338,9 +1338,12 @@ static void load_segments(struct vcpu *n) if ( n->arch.pv_vcpu.fs_base | (dirty_segment_mask & DIRTY_FS_BASE) ) wrfsbase(n->arch.pv_vcpu.fs_base); - /* Most kernels have non-zero GS base, so don't bother testing. */ - /* (This is also a serialising instruction, avoiding AMD erratum #88.) */ - wrmsrl(MSR_SHADOW_GS_BASE, n->arch.pv_vcpu.gs_base_kernel); + /* + * Most kernels have non-zero GS base, so don't bother testing. + * (For old AMD hardware this is also a serialising instruction, + * avoiding erratum #88.) + */ + wrgsshadow(n->arch.pv_vcpu.gs_base_kernel); /* This can only be non-zero if selector is NULL. */ if ( n->arch.pv_vcpu.gs_base_user | diff --git a/xen/arch/x86/hvm/vmx/vmx.c b/xen/arch/x86/hvm/vmx/vmx.c index 18d8ce2303..c7c8a0812f 100644 --- a/xen/arch/x86/hvm/vmx/vmx.c +++ b/xen/arch/x86/hvm/vmx/vmx.c @@ -503,12 +503,12 @@ static void vmx_save_guest_msrs(struct vcpu *v) * We cannot cache SHADOW_GS_BASE while the VCPU runs, as it can * be updated at any time via SWAPGS, which we cannot trap. */ - rdmsrl(MSR_SHADOW_GS_BASE, v->arch.hvm_vmx.shadow_gs); + v->arch.hvm_vmx.shadow_gs = rdgsshadow(); } static void vmx_restore_guest_msrs(struct vcpu *v) { - wrmsrl(MSR_SHADOW_GS_BASE, v->arch.hvm_vmx.shadow_gs); + wrgsshadow(v->arch.hvm_vmx.shadow_gs); wrmsrl(MSR_STAR, v->arch.hvm_vmx.star); wrmsrl(MSR_LSTAR, v->arch.hvm_vmx.lstar); wrmsrl(MSR_SYSCALL_MASK, v->arch.hvm_vmx.sfmask); @@ -2846,7 +2846,7 @@ static int vmx_msr_read_intercept(unsigned int msr, uint64_t *msr_content) break; case MSR_SHADOW_GS_BASE: - rdmsrl(MSR_SHADOW_GS_BASE, *msr_content); + *msr_content = rdgsshadow(); break; case MSR_STAR: @@ -3065,7 +3065,7 @@ static int vmx_msr_write_intercept(unsigned int msr, uint64_t msr_content) else if ( msr == MSR_GS_BASE ) __vmwrite(GUEST_GS_BASE, msr_content); else - wrmsrl(MSR_SHADOW_GS_BASE, msr_content); + wrgsshadow(msr_content); break; diff --git a/xen/arch/x86/pv/emul-priv-op.c b/xen/arch/x86/pv/emul-priv-op.c index ecb3b9c47b..af58544a3a 100644 --- a/xen/arch/x86/pv/emul-priv-op.c +++ b/xen/arch/x86/pv/emul-priv-op.c @@ -1032,7 +1032,7 @@ static int write_msr(unsigned int reg, uint64_t val, case MSR_SHADOW_GS_BASE: if ( is_pv_32bit_domain(currd) || !is_canonical_address(val) ) break; - wrmsrl(MSR_SHADOW_GS_BASE, val); + wrgsshadow(val); curr->arch.pv_vcpu.gs_base_user = val; return X86EMUL_OKAY; diff --git a/xen/arch/x86/x86_64/mm.c b/xen/arch/x86/x86_64/mm.c index 9b37da6698..2dfb6af5e9 100644 --- a/xen/arch/x86/x86_64/mm.c +++ b/xen/arch/x86/x86_64/mm.c @@ -1034,7 +1034,7 @@ long do_set_segment_base(unsigned int which, unsigned long base) case SEGBASE_GS_USER: if ( is_canonical_address(base) ) { - wrmsrl(MSR_SHADOW_GS_BASE, base); + wrgsshadow(base); v->arch.pv_vcpu.gs_base_user = base; } else diff --git a/xen/arch/x86/x86_64/traps.c b/xen/arch/x86/x86_64/traps.c index 4649ad4d10..4f85c32f92 100644 --- a/xen/arch/x86/x86_64/traps.c +++ b/xen/arch/x86/x86_64/traps.c @@ -49,7 +49,7 @@ static void read_registers(struct cpu_user_regs *regs, unsigned long crs[8]) regs->gs = read_sreg(gs); crs[5] = rdfsbase(); crs[6] = rdgsbase(); - rdmsrl(MSR_SHADOW_GS_BASE, crs[7]); + crs[7] = rdgsshadow(); } static void _show_registers( diff --git a/xen/include/asm-x86/msr.h b/xen/include/asm-x86/msr.h index b289b582f4..f14f265aa5 100644 --- a/xen/include/asm-x86/msr.h +++ b/xen/include/asm-x86/msr.h @@ -170,6 +170,22 @@ static inline unsigned long rdgsbase(void) return base; } +static inline unsigned long rdgsshadow(void) +{ + unsigned long base; + + if ( cpu_has_fsgsbase ) + { + asm volatile ( "swapgs" ); + base = __rdgsbase(); + asm volatile ( "swapgs" ); + } + else + rdmsrl(MSR_SHADOW_GS_BASE, base); + + return base; +} + static inline void wrfsbase(unsigned long base) { if ( cpu_has_fsgsbase ) @@ -194,6 +210,25 @@ static inline void wrgsbase(unsigned long base) wrmsrl(MSR_GS_BASE, base); } +static inline void wrgsshadow(unsigned long base) +{ + if ( cpu_has_fsgsbase ) + { + asm volatile ( "swapgs\n\t" +#ifdef HAVE_AS_FSGSBASE + "wrgsbase %0\n\t" + "swapgs" + :: "r" (base) ); +#else + ".byte 0xf3, 0x48, 0x0f, 0xae, 0xd8\n\t" + "swapgs" + :: "a" (base) ); +#endif + } + else + wrmsrl(MSR_SHADOW_GS_BASE, base); +} + DECLARE_PER_CPU(uint64_t, efer); static inline uint64_t read_efer(void) {